package com.gxljc.bear.crawler.itaogao.tmtpost;

import com.gxljc.commons.mongo.cli.BaseMongoRepository;
import com.gxljc.bear.crawler.base.BaseMongoTools;
import com.gxljc.bear.crawler.base.DolphinCrawlerConsts;
import com.gxljc.bear.crawler.base.DolphinFetchData;
import com.gxljc.bear.crawler.itaogao.ItaogaoConst;
import com.gxljc.bear.crawler.util.HtmlUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;

import java.io.Serializable;
import java.lang.reflect.Field;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 钛媒体PAGE数据爬虫。
 *
 * @author tanghaitao
 * @since 2022-8-4
 */
public class TmtpostTagCrawler implements Serializable {
	public static Logger LOG = Logger.getLogger(TmtpostTagCrawler.class);
	private Boolean proxy = false;

	public TmtpostTagCrawler(Boolean proxy) {
		this.proxy = proxy;
	}

	public void crawl() throws Exception {
		List<String>  seeds =  getSeed();
		if(CollectionUtils.isEmpty(seeds)){
			LOG.error("page is empty");
			return ;
		}
		crawl(seeds);
	}

	public void crawl(List<String>  seeds)  {
		if(CollectionUtils.isEmpty(seeds)){
			LOG.error("page is empty");
			return ;
		}
		for (String url : seeds){
			try {
				crawl(url);
			} catch (Exception e) {
				e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
			}
		}
	}

	//单条url入口
	public int crawl(String url) throws Exception {
		LOG.info("crawl = " + url);
		DolphinFetchData fetchData = DolphinFetchData.getInstance(DolphinCrawlerConsts.CrawlerChannelType.MOVIE.getName());
		byte[] htmlByte = fetchData.getHtml(url, proxy, ItaogaoConst.CRAWL_TIMEOUT);
		if (htmlByte == null) {
			LOG.error("page result is empty url = " + url);
			return DolphinCrawlerConsts.RESULT_NO;
		}
		String html = new String(htmlByte);
		if (StringUtils.isEmpty(html)) {
			LOG.error("page result is empty url = " + url);
			return DolphinCrawlerConsts.RESULT_NO;
		}
		List<TmtpostTagTable> tables = extract(html);
		saveMongodb(tables);
		return DolphinCrawlerConsts.RESULT_YES;
	}

	private List<TmtpostTagTable> extract(String html){
		Document doc = Jsoup.parse(html);
		Elements tagElement = doc.select(".btn-tag");
		if(CollectionUtils.isEmpty(tagElement)) return null;
		List<TmtpostTagTable> tables = new LinkedList<TmtpostTagTable>();
		for(Element tag : tagElement){
			try{
				String href = tag.attr("href");
				if(StringUtils.isEmpty(href)) continue;
				String id = extractId(href);
				String name = tag.text();
				String crawlUrl = HtmlUtil.pictureUrlPref(href,ItaogaoConst.TMTPOST_DOMAIN);
				TmtpostTagTable table = new TmtpostTagTable();
				table.setTagId(id);
				table.setName(name);
				table.setCrawlUrl(crawlUrl);
				tables.add(table);
			}catch (Exception er){
				er.printStackTrace();
			}
		}
		return tables;
	}

	//抽取id
	private String extractId(String href){

		final String patternUrl = "tag/(\\d+)";
		Pattern p = Pattern.compile(patternUrl);
		Matcher m = p.matcher(href);
		if (m.find()) {
			return m.group(1);
		}
		return null;
	}

	//save data to mongodb
	public void saveMongodb(List<TmtpostTagTable> tables) throws Exception {
		if(CollectionUtils.isEmpty(tables)) return ;
		for(TmtpostTagTable table : tables){
			saveMongodb(table);
		}
	}

	//save data to mongodb
	public void saveMongodb(TmtpostTagTable table) throws Exception {
		Query query = new Query();
		query.addCriteria(Criteria.where(DolphinCrawlerConsts.MONGODB_DEFAULT_ID).is(table.tagId));
		Update update = new Update();
		Field[] fields = table.getClass().getDeclaredFields();
		for (int i = 0; i < fields.length; i++) {
			Object value = fields[i].get(table);
			if (value == null) continue;
			if (value instanceof List) {
				if (CollectionUtils.isEmpty((List) value)) continue;
			}
			if (value instanceof String) {
				if (StringUtils.isEmpty((String) value)) continue;
			}
			update.set(fields[i].getName(), value);
		}
		BaseMongoRepository repo = BaseMongoTools.getInstance(DolphinCrawlerConsts.MongoDBName.ITAOGAO.getName());
		repo.upsert(query, update, ItaogaoConst.MONGODB_TMTPOST_TABLE_TAG);
	}

	//获取seed
	public List<String> getSeed() throws Exception {
		List<String> seeds = new LinkedList<String>();
		seeds.add("http://www.tmtpost.com/tag/all/A-D");
		seeds.add("http://www.tmtpost.com/tag/all/E-H");
		seeds.add("http://www.tmtpost.com/tag/all/I-L");
		seeds.add("http://www.tmtpost.com/tag/all/M-P");
		seeds.add("http://www.tmtpost.com/tag/all/Q-T");
		seeds.add("http://www.tmtpost.com/tag/all/U-X");
		seeds.add("http://www.tmtpost.com/tag/all/Y-Z");
		seeds.add("http://www.tmtpost.com/tag/all/other");
		return seeds;
	}
}
